package preprocessing;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Writer;
import java.io.BufferedWriter;
import java.io.OutputStreamWriter;
import java.util.regex.*;

/**
 * @author Thomas Wangler, wangler_thomas@hotmail.com
 * Translation Memories for patent translation
 * EnhTrainset
 * Class that enhances the metainformation in the corresponding metafiles and removes all sentences
 * from the corpus that are longer than 100 words. Any information is hardcoded
 * Please do not use without changing paths first.
 */
public class EnhTrainset {
	
	/**
	 * @param args
	 * @throws IOException 
	 */
	public static void main(String[] args) throws IOException {
		// TODO Auto-generated method stub

		//files
		String metafile = "D:\\pattr\\partitionedCorpus\\pattr.trainset.meta";
		String enfile = "D:\\pattr\\partitionedCorpus\\pattr.trainset.en";
		String defile = "D:\\pattr\\partitionedCorpus\\pattr.trainset.de";
		
		String trainfileEN = "D:\\pattr\\partitionedCorpus\\pattr.trainset.pre.en";
		String trainfileDE = "D:\\pattr\\partitionedCorpus\\pattr.trainset.pre.de";
		String trainfileMETA = "D:\\pattr\\partitionedCorpus\\pattr.trainset.pre.meta";
		
		String statfile = "D:\\pattr\\partitionedCorpus\\pattr.trainset.pre.stats";
		
		//streams
		FileInputStream metaStream = null;
		FileInputStream enStream = null;
		FileInputStream deStream = null;
		
		FileOutputStream trainENStream = null;
		FileOutputStream trainDEStream = null;
		FileOutputStream trainMETAStream = null;
		FileOutputStream trainSTATStream = null;
		
		
		try {
			
			trainENStream = new FileOutputStream(trainfileEN);
			trainDEStream = new FileOutputStream(trainfileDE);
			trainMETAStream = new FileOutputStream(trainfileMETA);
			trainSTATStream = new FileOutputStream(statfile);
			
			metaStream = new FileInputStream(metafile);
			enStream = new FileInputStream(enfile);
			deStream = new FileInputStream(defile);
			
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		}
	    
		

		
		Writer trainSetWriterEN = new BufferedWriter(new OutputStreamWriter(trainENStream, "UTF8"));
		Writer trainSetWriterDE = new BufferedWriter(new OutputStreamWriter(trainDEStream, "UTF8"));
		Writer trainSetWriterMETA = new BufferedWriter(new OutputStreamWriter(trainMETAStream, "UTF8"));
		Writer trainSetWriterSTAT = new BufferedWriter(new OutputStreamWriter(trainSTATStream, "UTF8"));
		
		BufferedReader metareader = new BufferedReader(new InputStreamReader(metaStream, "UTF8"));
	    BufferedReader enreader = new BufferedReader(new InputStreamReader(enStream, "UTF8"));
	    BufferedReader dereader = new BufferedReader(new InputStreamReader(deStream, "UTF8"));
	
	    String metaline = metareader.readLine();
	    String enline = enreader.readLine();
	    String deline = dereader.readLine();
        
	    
	    Pattern catpattern = Pattern.compile(".+\t.+\t.+\t(.).+");
	    
	    while(metaline != null && enline != null && enline != null){

	    	String cat = "";
	    	
	    	Matcher catmatch = catpattern.matcher(metaline);
	    	if(catmatch.matches()){
	    		cat = cat + catmatch.group(1);
	    	}
	    	
	    	int delen = deline.split(" ").length;
	    	int enlen = enline.split(" ").length;
	    	
	    	if(delen < 100){

	    		trainSetWriterEN.write(enline + "\n");
	    		trainSetWriterDE.write(deline + "\n");
	    		trainSetWriterMETA.write(metaline + "\t" + delen + "\n");
	    		trainSetWriterSTAT.write(delen + "\t" + enlen + "\t" + cat + "\n");
	    		
	    	}else{
	    		trainSetWriterSTAT.write("deleted: " + delen + "\n");
	    	}
            
	    	//advancing Readers
	    	metaline = metareader.readLine();
            enline = enreader.readLine();
            deline = dereader.readLine();

        }
	    

		trainENStream.close();
		trainDEStream.close();
		trainMETAStream.close();
		trainSTATStream.close();
		metaStream.close();
		enStream.close();
		deStream.close();
	    
	    
	}

}

